var async = require('async');
var fn = require('./fn.js');
var EventProxy = require('eventproxy');
var express = require('express');
var geturls = require("./fxurls.js");
var results = new Array();
var endurls = new Array();
var oldurls = new Array();
var nn = new Array();
var oldurl = "";
var oldoo = "";
var oo = "";
var oldooo = "";
var s_url = "";
var count=0;
var isgbk="";
var bianma="gbk";

function profile(url,gather_type,callback) {
    clean();
    arr=new Array();
    if(gather_type==6){
        dogeturls(url,function cb(result){
            arr=result;
            do2(arr);
        });
    }else{
        do2(arr);
    }
    function do2(arr){
        if (arr != undefined && arr.length < 1) {
            re = /,/gi;
            re2 = /\[(\d+)-(\d+)\]/gi;
            if (re.test(url)) {
                arr = fn.preg_split(",", url);
            } else if ((nn = re2.exec(url)) != null) {
                arr = new Array();
                for (i = nn[1]; i < nn[2]; i++) {
                    arr.push(url.replace(re2, i));
                }
            } else {
                arr = new Array();
                arr.push(url);
            }
        }
        getnews(arr);
        proxy.after("isend", 1, function (isend) {
            callback(isend[(isend.length - 1)])
        });
    }
}

function dogeturls(url,cb){
    geturls.create(url,function(results){
        cb(geturls.themax());
    });
}

function getnews(arr) {
    async.each(arr, function (item, done) {
        async.series([
            function (cb) {
                if (item == undefined){
                    cb(new Error('error'), 0);
                }else {
                    console.log(">>>>" + item);
                    count++;
                    s_url = fn.trim(item);
                    filename = "d:\\temp\\" + item.replace(/\/|\:|\?|\=|\&|\-|%|;/gi, "").replace(/\//, "\\").substring(0, 200);
                    filename = filename.trim();
                    cb();
                }
            },
            function (cb) {
                //如果本地没有该网页的静态文件，抓取之，并保存成静态文件
                filename = "d:\\temp\\" + item.replace(/\/|\:|\?|\=|\&|\-|%|;|-/gi, "").replace(/\//,"\\").substring(0, 200);
                filename = filename.trim();
                var fs = require("fs");
                if (!fs.existsSync(filename)) {
                    exports.host = fn.gethost(item);
                    exports.host2 = fn.gethost2(item);
                    nextpagerem = "";
                    rssql = require("./zqnewsmysql");
                    rssql.getrs(exports.host, function (rs) {
                        fn.getwrite(item,rs,function(data){
                            exports.html = data;
                            cb();
                        });
                    },exports.host2);
                }else{
                    var iconv = require('iconv-lite');
                    var fileStr = fs.readFileSync(filename, {encoding:'binary'});
                    var buf = new Buffer(fileStr, 'binary');
                    var str = iconv.decode(buf, 'utf8');
                    exports.html = str;
                    cb();
                }
            },
            function (cb) {
                //ng.get("http://collect2.longsunhd.com/zqnews_api.php?url=" + encodeURIComponent(item), function (data, status, headers) {
                //    console.log("获得" + item );
                //    if (data.substring(0, 1) != "[" && data.substring(0, 1) != "{") {
                //        console.log("" + item + " time out!");
                //        count--;
                //    } else {
                //        result = JSON.parse(data);
                //        if (urlcontinue(item, s_url, s_url, result) && status == 200) {
                //            if (!fn.in_array(item, oldurls)) {
                //                oldurls.push(item);
                //            }
                //            if (fn.is_array(results)) results.push(result);
                //            if (result.nextPage != "" && result.nextPage != null && !result.nopage) {
                //                //console.log(result.nextPage);
                //                var arr2 = new Array();
                //                arr2[0] = result.nextPage;
                //                getnews(arr2);
                //            } else {
                //                cb();
                //            }
                //        }else{
                //            count--;
                //        }
                //    }
                //}, header, 'utf8').on('error', function (e) {
                //    console.log("error:" + item + " " + e.message);
                //    count--;
                //});
                fxnews=require("./fxnews.js");
                oldurls.push(item);
                fxnews.fxnews(item,comeback=function(result){
                    results[results.length]=result;
                    //console.log(">>>>---has:"+result.message);
                    if (result.nextPage != "" && result.nextPage != null && !result.nopage&&!fn.in_array(result.nextPage, oldurls)&&result.nextPage.match(/\/\\/)==null) {
                        oldurls.push(result.nextPage);
                        fxnews.fxnews(result.nextPage,comeback);
                    }else {
                        cb();
                    }
                });
            },function(cb){
                console.log("" + item + " 完成！ "+results.length+" "+count);
                if(results.length>=count){
                    data = zhenghe(results);
                    results = JSON.stringify(data);
                    proxy.emit("isend", results);
                    //console.log(results);
                    clean();
                }
            }
        ], function (err, values) {
            console.log(item+" 抓取失败!");
        });

    }, function (err) {
        if (err) throw err;
        data = zhenghe(results);
        results = JSON.stringify(data);
        proxy.emit("isend", results);
        //console.log(results);
        clean();
    });
}

function clean() {
    results = [];
    result = [];
    endurls = [];
    oldurls = [];
    oldurl = "";
    oldoo = "";
    oldooo = "";
    count=0;
}

function zhenghe(results) {
    newarr = new Array();
    for (s = 0; s < results.length; s++) {
        if (!fn.in_array(results[s].subject, getsubject(newarr))) {
            newarr.push(results[s]);
        }else {
            i = getindexbytitle(results[s].subject, newarr);
            newarr[i].message = newarr[i].message + "\r\n" + results[s].message;
        }
    }
    return newarr;
}


function getsubject(sarr) {
    returnarr = new Array();
    for (s2 = 0; s2 < sarr.length; s2++) {
        if (!fn.in_array(sarr[s2].subject, returnarr)) {
            returnarr.push(sarr[s2].subject);
        }
    }
    return returnarr;
}

function getindexbytitle(title, newarr) {
    for (s3 = 0; s3 < newarr.length; s3++) {
        onestr = newarr[s3].subject.toString();
        if (onestr == title) {
            return s3;
            break;
        }
    }
    return 0;
}

function urlcontinue(item, url, baseurl, result) {
    if (is_root(item))  return false;
    item = quanurl(trim(item), url, baseurl);
    if (preg_match("^404|403", result.subject) || fn.in_array(item, oldurls) || preg_match("([0-9_\-]+)??(_|\-)([0-9]+)\.htm", item, oo) && oldurl != "" && oo[1] != "" && oldoo != oo[1] && oldooo != oo[3]) {
        oldurl = "";
        oldoo = "";
        oldooo = "";
        return false;
    } else {
        oldurl = item;
        oldoo = oo[1];
        oldooo = oo[3];
    }
    return true;
}


collect = function (req, res) {
    var domain = require('domain');
    var reqd = domain.create();
    reqd.add(req);
    reqd.add(res);
    reqd.on('error', function (er) {
        console.error('Error', er, req.url);
        try {
            res.writeHead(500);
            res.end('Error occurred, sorry.');
        } catch (er) {
            console.error('Error sending 500', er, req.url);
        }
    });
    s_url = (req.body.s_url != ""&&req.body.s_url!=undefined) ? req.body.s_url : req.query.s_url;
    gather_type = (req.body.gather_type != ""&&req.body.gather_type!=undefined) ? req.body.gather_type : req.query.gather_type;
    profile(s_url,gather_type, function (data) {
        res.send("comeback2("+data+")");
    });
}

// http://127.0.0.1:8080/collect/?s_url=http://news.gxnews.com.cn/staticpages/20150410/newgx552700a4-12562577.shtml
//http://127.0.0.1:8080/collect/?s_url=http://politics.scdaily.cn/mzfz/201504/10126965.html
//http://politics.scdaily.cn/mzfz/201504/10126965.html
var proxy = new EventProxy();
var app = express();
var bodyParser = require('body-parser');
app.use(bodyParser.urlencoded({extended: false}));
app.use(bodyParser.json());
app.get('/collect/*', collect);
app.post('/collect/*', collect);
//var router = express.Router();
//router.get('/collect/*', collect);
//router.post('/collect2/*', collect2);
app.listen(8080);
console.log("listen to 8080");
//profile("http://politics.scdaily.cn/mzfz/201504/10126965.html",function(data){
//});


